package util;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.Reader;
import java.io.IOException;

import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.LengthFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;

public class WordCleaner {
	private File dir;
	private String[] allFileNames;

	public WordCleaner(String path) {
		dir = new File(path);
		allFileNames = dir.list();
		if (allFileNames == null) {
			System.out
					.println("WordCleaner: does not exist or is not a directory");
			System.exit(0);
		}
	}

	// pass in one full file path
	@SuppressWarnings("deprecation")
	public void cleanFiles() throws IOException {
		PorterStemFilter psFilter = null;
		
	    File targetDirFile = new File("cleaned");
	    
	    if (targetDirFile.exists()) {
	      try {
			FileUtils.forceDelete(targetDirFile);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	    }
	    try {
			FileUtils.forceMkdir(targetDirFile);
		} catch (IOException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}

		try {
			for (int i = 0; i < allFileNames.length; i++) {
				Reader reader = new FileReader(dir + "/" + allFileNames[i]);

				StandardTokenizer stdTokenizer = new StandardTokenizer(
						Version.LUCENE_35, reader);
				LowerCaseFilter lcFilter = new LowerCaseFilter(stdTokenizer);
				StopFilter stopFilter = new StopFilter(true, lcFilter,
						StopAnalyzer.ENGLISH_STOP_WORDS_SET);
				LengthFilter lenFilter = new LengthFilter(stopFilter, 2, 10);
				psFilter = new PorterStemFilter(lenFilter);

				try {
					FileWriter fstream = new FileWriter("cleaned/cleaned_"
							+ allFileNames[i]);
					BufferedWriter out = new BufferedWriter(fstream);
					CharTermAttribute charTermAttribute = psFilter
							.getAttribute(CharTermAttribute.class);

					while (psFilter.incrementToken()) {
						String term = charTermAttribute.toString();
						term.replaceAll("[^\\p{L}\\p{N}]", "");
						boolean numFlag = false;
						for(int j = 0; j < term.length(); j++) {
							char c = term.charAt(j);
							if(Character.isDigit(c)) {
								numFlag = true;
							}
						}
						if((!numFlag) && (!term.contains(".")) && (!term.contains("_"))) {
							out.write(term);
							out.newLine();
						}
					}
					out.close();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}	
	}
}